'use strict'

const Crawler = require('crawler');

var url = 'https://book.douban.com/top250?start=0';
var startPage = 0;
var endPage = 225;
var arr = [0, 25, 50, 75, 100, 125, 150, 175, 200, 225];
var int = 1;
var xls = 0;

const crawler = new Crawler({
    maxConnections: 10,
    // This will be called for each crawled page
    callback: function (error, res, done) {
        if (error) {
            console.log(error);
        } else {
            var $ = res.$;
            // $ is Cheerio by default
            //a lean implementation of core jQuery designed specifically for the server
            //var list=$('#content .article .paginator ').toArray();
            //console.log(list);
            var list = $('#content .article table').toArray();
            // console.log('Grabbed', $('title').text(), 'bytes');
            // console.log(list.length);
            if (startPage <= endPage) {
                list.forEach(element => {
                    var txt = $(element);
                    console.log(txt.find('.pl2 a').text().trim().replace(/[ \n\r]/g, ''));
                });
                url = 'https://book.douban.com/top250?start=' + arr[int++];
                startPage += list.length
                xls+=list.length
                crawler.queue(url);
            }
        }
        console.log(xls)
        console.log(url);
        done();
       
    }
});

crawler.queue(url)




// crawler.queue([{
//     uri: url,
//     jQuery: true,
//     // The global callback won't be called
//     callback: function (error, res, done) {
//         if (error) {
//             console.log(error);
//         } else {
//             var $ = res.$;
//             var list=$('#content .article table').toArray();
//             console.log('Grabbed',$('title').text(), 'bytes');
//             console.log(list.length);
//             if(startPage<=endPage){
//                 var int=1;
//                 list.forEach(element => {
//                     var txt=$(element);
//                     console.log(txt.find('.pl2 a').text().trim().replace(/[ \n\r]/g,''));
//                 });
//                 var surl='https://book.douban.com/top250?start='+arr[int++];
//                 crawler.queue(surl);
//                 console.log(surl);
//             }
//         }
//         done();
//     }
// }]);